package com.aerfafish.spider.spider;

import com.aerfafish.spider.callback.SpiderCallback;
import com.aerfafish.spider.neo4j.dao.Share;
import com.aerfafish.spider.utils.RequestUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.scheduler.component.DuplicateRemover;

import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author aerfafish
 * @date 2021/4/19 4:13 下午
 */
public class ShareSpider implements PageProcessor {

    private final static Logger logger = LoggerFactory.getLogger(ShareSpider.class);

    private Spider spider = Spider.create(this).setExitWhenComplete(false).thread(10);

    private Map<String, Share> shareMap = new ConcurrentHashMap<>();

    private SpiderCallback spiderCallback;

    private static ExecutorService executorService = Executors.newSingleThreadExecutor();

    public ShareSpider(SpiderCallback spiderCallback) {
        // http://push2.eastmoney.com/api/qt/stock/get?secid=105.TSLA&ut=bd1d9ddb04089700cf9c27f6f7426281&fields=f43,f169,f170,f46,f60,f84,f116,f44,f45,f171,f126,f47,f48,f168,f164,f49,f161,f55,f92,f59,f152,f167,f50,f86,f71,f172&cb=jQuery1124006878735018143023_1621419039861&type=CT&cmd=TSLA7&sty=FDPBPFB&st=z&js=((x))&token=4f1862fc3b5e77c150a2b985b12db0fd&_=1621419039862

        this.spiderCallback = spiderCallback;
        spider.setScheduler(new QueueScheduler().setDuplicateRemover(new DuplicateRemover() {
            @Override
            public boolean isDuplicate(Request request, Task task) {
                return false;
            }

            @Override
            public void resetDuplicateCheck(Task task) {

            }

            @Override
            public int getTotalRequestsCount(Task task) {
                return 0;
            }
        }));
        spider.start();
    }


    @Override
    public void process(Page page) {
        if (page.getUrl().regex("^http\\://quote\\.eastmoney\\.com/concept/").match()) {
            String shareCode = page.getUrl().regex("(^http\\://quote\\.eastmoney\\.com/concept/)(.*)(\\.html\\?from=classic$)", 2).toString();
            if (shareCode == null) {
                logger.error("shareCode not match");
                return;
            }
            parseHtml(shareCode, page.getRawText());
            spiderUrl(shareCode);
        } else if (page.getUrl().regex("^http\\://quote\\.eastmoney\\.com/newstatic/js/concept\\.js$").match()) {
            Pattern pattern = Pattern.compile("^http://quote\\.eastmoney\\.com/concept/(.*)\\.html\\?from=classic$");
            Matcher matcher = pattern.matcher(page.getRequest().getHeaders().get("Referer"));
            String shareCode = null;
            if (matcher.find()) {
                shareCode = matcher.group(1);
            }
            if (shareCode == null) {
                logger.error("parse js get shareCode error");
                return;
            }
            parseJs(shareCode, page.getRawText());
        } else if (page.getUrl().regex("^http://push2\\.eastmoney\\.com/api/qt/stock/get").match()) {
            String shareCode = page.getUrl().regex("code=(.*)", 1).toString();
            parseShareData(shareCode, page.getRawText());
        } else if (page.getUrl().toString().equals("http://quote.eastmoney.com/newstatic/js/hk.js")) {
            Pattern pattern = Pattern.compile("^http://quote\\.eastmoney\\.com/(.*)\\.html$");
            Matcher matcher = pattern.matcher(page.getRequest().getHeaders().get("Referer"));
            String shareCode = null;
            if (matcher.find()) {
                shareCode = matcher.group(1);
            }
            if (shareCode == null) {
                logger.error("parse js get shareCode error");
                return;
            }
            parseHKJs(shareCode, page.getRawText());
        } else if (page.getUrl().toString().equals("http://quote.eastmoney.com/newstatic/js/us.js")) {
            Pattern pattern = Pattern.compile("^http://quote\\.eastmoney\\.com/(.*)\\.html$");
            Matcher matcher = pattern.matcher(page.getRequest().getHeaders().get("Referer"));
            String shareCode = null;
            if (matcher.find()) {
                shareCode = matcher.group(1);
            }
            if (shareCode == null) {
                logger.error("parse js get shareCode error");
                return;
            }
            parseUSJs(shareCode, page.getRawText());
        } else if (page.getUrl().regex("^http\\://quote\\.eastmoney\\.com/hk").match()) {
            String shareCode = page.getUrl().regex("(^http\\://quote\\.eastmoney\\.com/)(.*)(\\.html$)", 2).toString();
            parseHKHtml(shareCode, page.getRawText());
            spiderUrl(shareCode);
        } else if (page.getUrl().regex("^http\\://quote\\.eastmoney\\.com/us").match()) {
            String shareCode = page.getUrl().regex("(^http\\://quote\\.eastmoney\\.com/)(.*)(\\.html$)", 2).toString();
            parseUSHtml(shareCode, page.getRawText());
            spiderUrl(shareCode);
        }
    }

    @Override
    public Site getSite() {
        return Site.me().setCycleRetryTimes(3000).setSleepTime(10).setCharset("UTF-8").setTimeOut(20000)
                .setUserAgent("Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36");
    }

    public void spiderShare(String shareCode) {
        spiderStockEntry(shareCode);
    }

    public void spiderStockEntry(String shareCode) {
        if (shareCode.contains("hk/")) {
            spider.addRequest(RequestUtil.urlToRequest(
                    String.format("http://quote.eastmoney.com/%s.html", shareCode),
                    String.format("http://quote.eastmoney.com/%s.html", shareCode)));

        } else if (shareCode.contains("us/")) {
            spider.addRequest(RequestUtil.urlToRequest(
                    String.format("http://quote.eastmoney.com/%s.html", shareCode),
                    String.format("http://quote.eastmoney.com/%s.html", shareCode)));
        } else {
            spider.addRequest(RequestUtil.urlToRequest(
                    String.format("http://quote.eastmoney.com/concept/%s.html?from=classic", shareCode),
                    String.format("http://quote.eastmoney.com/%s.html", shareCode)));
        }
    }

    public void spiderUrl(String shareCode) {
        if (shareCode.contains("hk/")) {
            spider.addRequest(RequestUtil.urlToRequest(
                    "http://quote.eastmoney.com/newstatic/js/hk.js",
                    String.format("http://quote.eastmoney.com/%s.html", shareCode)));

        } else if (shareCode.contains("us/")) {
            spider.addRequest(RequestUtil.urlToRequest(
                    "http://quote.eastmoney.com/newstatic/js/us.js",
                    String.format("http://quote.eastmoney.com/%s.html", shareCode)));
        } else {
            spider.addRequest(RequestUtil.urlToRequest(
                    "http://quote.eastmoney.com/newstatic/js/concept.js",
                    String.format("http://quote.eastmoney.com/concept/%s.html?from=classic", shareCode)));
        }


    }

    private void parseHtml(String shareCode, String data) {
        try {
            Share share = new Share();
            Pattern pattern = Pattern.compile("(var stockentry = )(\\{[^;]*})");
            Matcher matcher = pattern.matcher(data);
            if (matcher.find()) {
                String stockEntry = matcher.group(2);
                if (stockEntry == null || stockEntry.equals("")) {
                    logger.error("parse stockEntry error");
                    return;
                }
                JSONObject stockEntryObject = JSON.parseObject(stockEntry);
                share.setCode(stockEntryObject.getString("code"));
                share.setShareName(stockEntryObject.getString("name"));
                share.setMarketNum(Integer.parseInt(stockEntryObject.getString("marketnum")));
                share.setMarket(stockEntryObject.getString("shortmarket"));
                shareMap.put(shareCode, share);
            }
        } catch (Exception e) {
            logger.error("", e);
        }
    }

    private void parseUSHtml(String shareCode, String data) {
        try {
            Share share = new Share();
            Pattern pattern = Pattern.compile("(var stockEnity = )(\\{[^}]*})");
            Matcher matcher = pattern.matcher(data);
            if (matcher.find()) {
                String stockEntry = matcher.group(2);
                if (stockEntry == null || stockEntry.equals("")) {
                    logger.error("parse stockEntry error");
                    return;
                }
                JSONObject stockEntryObject = JSON.parseObject(stockEntry);
                share.setCode(stockEntryObject.getString("stockCode"));
                share.setShareName(stockEntryObject.getString("stockName"));
                share.setMarketNum(Integer.parseInt(stockEntryObject.getString("MktNum")));
                share.setMarket("us/");
                shareMap.put(shareCode, share);
            }
        } catch (Exception e) {
            logger.error("", e);
        }
    }

    private void parseHKHtml(String shareCode, String data) {
        try {
            Share share = new Share();
            //var stockcode = "00345", stockmarket = "5", stockname="VITASOY INT&#39;L", MktNum="116";
            Pattern pattern = Pattern.compile("(var stockcode = [^\n]*;)");

            Matcher matcher = pattern.matcher(data);
            if (matcher.find()) {
                String stockEntry = matcher.group(1);
                if (stockEntry == null || stockEntry.equals("")) {
                    logger.error("parse stockEntry error");
                    return;
                }
                Pattern codePattern = Pattern.compile("(stockcode = )\"([^\"]*)\"");
                Pattern marketPattern = Pattern.compile("(MktNum=)\"([^\"]*)\"");
                Pattern namePattern = Pattern.compile("(stockname=)\"([^\"]*)\"");
                Matcher codeMatcher = codePattern.matcher(stockEntry);
                Matcher marketMatcher = marketPattern.matcher(stockEntry);
                Matcher nameMatcher = namePattern.matcher(stockEntry);

                if (codeMatcher.find() && marketMatcher.find() && nameMatcher.find()) {
                    share.setCode(codeMatcher.group(2));
                    share.setShareName(nameMatcher.group(2));
                    share.setMarket("hk/");
                    share.setMarketNum(Integer.parseInt(marketMatcher.group(2)));
                } else {
                    logger.error("parse " + stockEntry + " error");
                }
                shareMap.put(shareCode, share);
            }
        } catch (Exception e) {
            logger.error("", e);
        }
    }

    private void parseJs(String shareCode, String data) {
        try {
            Share share = shareMap.get(shareCode);
            String url = "http://push2.eastmoney.com/";
            Pattern pattern = Pattern.compile("var url = pathConfig\\.getEnvPath\\(\"commonApi\"\\) \\+ \'([^\']*)\'\\s*" +
                    "\\+\'([^\']*)\'\\s*" +
                    "\\+\'([^\']*)\'\\s*" +
                    "\\+\'([^\']*)\'\\s*" +
                    "\\+\'([^\']*)\'\\s*" +
                    "\\+\'([^\']*)\'\\s*" +
                    "\\+\'([^\']*)\'");
            Matcher matcher = pattern.matcher(data);
            if (matcher.find()) {
                for (int i = 1; i <= matcher.groupCount(); i++) {
                    url = url.concat(matcher.group(i));
                }
            } else {
                logger.error("match js to get url error");
                return;
            }
            url = url + (share.getMarketNum() == 1?"1":"0") + "." + share.getCode();
            url = url + "&cb=" + shareCode;
            url = url + "&code=" + shareCode;
            Map<String, String> headers = new HashMap<>();
            headers.put("Referer", "http://quote.eastmoney.com/");
            spider.addRequest(RequestUtil.urlToRequest(url, headers));
        } catch (Exception e) {
            logger.error("", e);
        }
    }

    private void parseHKJs(String shareCode, String data) {
        try {
            Share share = shareMap.get(shareCode);
            String url = "http://push2.eastmoney.com/api/qt/stock/get?";
            Pattern pattern = Pattern.compile("var par = (\\{[^}]+})");
            Matcher matcher = pattern.matcher(data);
            if (matcher.find()) {
                String urlEntry = matcher.group(1);
                Pattern secidPattern = Pattern.compile("(secid: )\"([^\"]*)\"");
                Pattern flttPattern = Pattern.compile("(fltt: )([0-9]*)");
                Pattern utPattern = Pattern.compile("(ut: )\"([^\"]*)\"");
                Pattern invtPattern = Pattern.compile("(invt: )([0-9]*)");
                Pattern fieldsPattern = Pattern.compile("(fields: )\"([^\"]*)\"");
                Matcher secidMatcher = secidPattern.matcher(urlEntry);
                Matcher flttMatcher = flttPattern.matcher(urlEntry);
                Matcher utMatcher = utPattern.matcher(urlEntry);
                Matcher invtMatcher = invtPattern.matcher(urlEntry);
                Matcher fieldsMatcher = fieldsPattern.matcher(urlEntry);
                if (secidMatcher.find() && flttMatcher.find() && utMatcher.find() && invtMatcher.find() && fieldsMatcher.find()) {
                    url += "secid=" + secidMatcher.group(2)+share.getCode();
                    url += "&fields=" + fieldsMatcher.group(2);
                    url += "&ut=" + utMatcher.group(2);
                    url += "&fltt=" + flttMatcher.group(2);
                    url += "&invt=" + invtMatcher.group(2);
                    url += "&cb=" + "jQuery.jQuery" + "6930140582750544" + "_"+System.currentTimeMillis();
                    url += "&_=" + System.currentTimeMillis();
                    url += "&code=" + shareCode;
                } else {
                    logger.error("parse " + urlEntry + " error");
                }

                Map<String, String> headers = new HashMap<>();
                headers.put("Referer", "http://quote.eastmoney.com/");
                spider.addRequest(RequestUtil.urlToRequest(url, headers));

            } else {
                logger.error("match js to get url error");
                return;
            }
//            url = url + (share.getMarketNum() == 1?"1":"0") + "." + share.getCode();
//            url = url + "&cb=" + shareCode;

        } catch (Exception e) {
            logger.error("", e);
        }
    }

    private void parseUSJs(String shareCode, String data) {
        try {
            Share share = shareMap.get(shareCode);
            String url = "http://push2.eastmoney.com/api/qt/stock/get?";

            Pattern utPattern = Pattern.compile("(Usstock.prototype.ut = )'([^']*)'");
            Matcher matcher = utPattern.matcher(data);
            if (matcher.find()) {
                String ut = matcher.group(2);
                url += "secid=" + share.getMarketNum()+"."+share.getCode();
                url += "&ut=" + ut;
                url += "&fields=f43,f169,f170,f46,f60,f84,f116,f44,f45,f171,f126,f47,f48,f168,f164,f49,f161,f55,f92,f59,f152,f167,f50,f86,f71,f172";
                url += "&cb=jQuery112402659512525693888_" + System.currentTimeMillis();
                url += "&_=" + System.currentTimeMillis();
                url += "&code=" + shareCode;
            }
            Map<String, String> headers = new HashMap<>();
            headers.put("Referer", "http://quote.eastmoney.com/");
            spider.addRequest(RequestUtil.urlToRequest(url, headers));

        } catch (Exception e) {
            logger.error("", e);
        }
    }

    // f43
    public void parseShareData(String shareCode, String shareData) {
        Share share = shareMap.get(shareCode);
        Pattern pattern = Pattern.compile("\\((.*)\\);$");
        Matcher matcher = pattern.matcher(shareData);
        try {
            if (matcher.find()) {
                String data = matcher.group(1);
                JSONObject dataObject = JSON.parseObject(data).getJSONObject("data");
                if (dataObject == null) {
                    logger.error("dataObject is null");
                    return;
                }
                try {
                    Float value = dataObject.getFloat("f43");
                    Float incRate = dataObject.getFloat("f170");
                    if (shareCode.contains("us/")) {
                        value /= 100;
                        incRate /= 100;
                    }

                    share.setLatestValue(value);
                    share.setIncRate(incRate);
                } catch (Exception e) {
                    share.setLatestValue(0F);
                    share.setIncRate(0F);
                }

            } else {
                logger.error("parse share data error");
                return;
            }
            spiderCallback.onSaveShare(share);
        } catch (Exception e) {
            logger.error("", e);
        } finally {
            shareMap.remove(shareCode);
        }
    }
}
